package evaluation;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileFilter;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;

public class DatabaseSortBySizeTfIdfCleaner {

	/**
	 * @param args
	 * @throws IOException 
	 */
	public static void main(String[] args) throws IOException {
		
		String type = "training";
		
		int numberoftfIdf = 50;
		
		File folder = new File("/proj/dbNoBackup/pjbarrio/Experiments/Wrappers/SizeSortedList/" + type);
		
		File[] files = folder.listFiles(new FileFilter() {
			
			@Override
			public boolean accept(File pathname) {
				return pathname.getName().endsWith(".list");
			}
		});
		
		for (int i = 0; i < files.length; i++) {
			
			File newout = new File(folder,files[i].getName() + ".clean");
			
			BufferedReader br = new BufferedReader(new FileReader(files[i]));
			
			String line;
			
			BufferedWriter bw = new BufferedWriter(new FileWriter(newout));
			
			while ((line=br.readLine())!=null){
				
				int nam = Integer.valueOf(line.replace(".html", ""));
				
				if (nam < numberoftfIdf){
					bw.write(line + "\n");
				}
				
			}
			
			bw.close();
			
			br.close();
			
		}
		
	}

}
